/**
 * 
 */
package edu.umd.clip.lm.model;

import org.w3c.dom.*;
import org.w3c.dom.bootstrap.DOMImplementationRegistry;
import org.w3c.dom.ls.DOMImplementationLS;
import org.w3c.dom.ls.LSOutput;
import org.w3c.dom.ls.LSSerializer;
import org.xml.sax.SAXException;

import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.Environment;

import java.io.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.*;
import java.util.*;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import edu.umd.clip.jobs.*;
import edu.umd.clip.lm.storage.*;
import edu.umd.clip.lm.storage.remote.ClientStorageConnection;
import edu.umd.clip.lm.util.tree.*;
import edu.umd.clip.lm.util.*;
import edu.umd.clip.lm.model.data.Context;
import edu.umd.clip.lm.model.decoding.*;
import edu.umd.clip.lm.model.training.metrics.*;
/**
 * @author Denis Filimonov <den@cs.umd.edu>
 *
 */
public class LanguageModel extends XMLizable {
	private static final long serialVersionUID = 1L;
	
	public static final String XML_ELEMENT_NAME = "language-model";
	public static final String XML_ATTR_ORDER_NAME = "order";
	public static final String XML_ATTR_OVERT_ORDER_NAME = "overt-order";
	public static final String XML_ATTR_HIDDEN_ORDER_NAME = "hidden-order";
	public static final String XML_ATTR_ID_NAME = "id";
	public static final String PRIMARY_LM_ID = "primary"; // the main LM
	public static final String XML_BACKOFF_LM_NAME = "backoff-forest";
	public static final String XML_HISTORY_TREE_NAME = "history-tree";
	private static final String XML_USE_COMPLEX_QUESTIONS_NAME = "use-complex-questions";
	private static final String XML_USE_HFT_QUESTIONS_NAME = "use-hft-questions";
	private static final String XML_MIN_TOTAL_COUNT_NAME = "min-total-count";
	private static final String XML_MIN_COST_DECREASE_NAME = "min-cost-decrease";
	private static final String XML_MAX_CLUSTERS_NAME = "max-clusters";
	private static final String XML_OVERT_QUESTION_PENALTY_NAME = "overt-question-penalty";
	private static final String XML_FORCE_HIDDEN_QUESTIONS_NAME = "force-hidden-questions";
	private static final String XML_PRUNING_THRESHOLD_NAME = "pruning-threshold";
	private static final String XML_CONTEXT_VAR_METRICS_NAME = "ctx-var-metrics";
	private static final String XML_CONTEXT_VAR_STOPPING_RULE_NAME = "ctx-var-stopping-rule";
	private static final String XML_RANDOMNESS_NAME = "randomness";
	
	private long MIN_TOTAL_COUNT = 40;
	private double MIN_COST_DECREASE = 1e-7;

	private byte order = 3;
	private byte overtOrder = 3;
	private byte hiddenOrder = 3;
	private String id = PRIMARY_LM_ID;
	private int idNum = -1;
	//private String backoffLM = null;
	private BinaryTree<HistoryTreePayload> historyTree;
	//private Decoder decoder = null;
	
	private boolean useComplexQuestions = false;
	private boolean useHFTQuestions = true;
	private double overtQuestionPenalty = 0.0;
	private int forceHiddenQuestions = 0;
	private int maxClusters = Integer.MAX_VALUE;
	private int pruningThreshold = 0;
	private float randomness = 0f;

	private ContextVariableMetrics contextVariableMetrics;
	private ContextVariableStoppingRule contextVariableStoppingRule;
	
	public LanguageModel(String name, byte overtOrder, byte hiddenOrder) {
		this.id = name;
		this.overtOrder = overtOrder;
		this.hiddenOrder = hiddenOrder;
		this.order = (byte) Math.max(overtOrder, hiddenOrder);
	}
	
	@SuppressWarnings("unchecked")
	public LanguageModel(Element xmlnode) {
		assert(xmlnode.getTagName().equals(XML_ELEMENT_NAME));
		String attr = xmlnode.getAttribute(XML_ATTR_ORDER_NAME);
		if (attr != null && !attr.equals("")) {
			order = Byte.parseByte(attr);
			hiddenOrder = order;
			overtOrder = order;
		}
		
		attr = xmlnode.getAttribute(XML_ATTR_OVERT_ORDER_NAME);
		if (!attr.equals("")) {
			overtOrder = Byte.parseByte(attr);
		}
		
		attr = xmlnode.getAttribute(XML_ATTR_HIDDEN_ORDER_NAME);
		if (!attr.equals("")) {
			hiddenOrder = Byte.parseByte(attr);
		}
		
		order = (byte)Math.max(hiddenOrder, overtOrder);
		
		attr = xmlnode.getAttribute(XML_ATTR_ID_NAME);
		if (!attr.equals("")) {
			id = attr;
		}
		
		Element elem;
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode, XML_USE_COMPLEX_QUESTIONS_NAME);
		if (elem != null) {
			useComplexQuestions = Boolean.parseBoolean(elem.getTextContent());
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode, XML_USE_HFT_QUESTIONS_NAME);
		if (elem != null) {
			useHFTQuestions = Boolean.parseBoolean(elem.getTextContent());
		}

		elem = XMLUtils.getFirstElementByTagName(xmlnode,XML_MIN_TOTAL_COUNT_NAME);
		if (elem != null) {
			MIN_TOTAL_COUNT = Long.parseLong(elem.getTextContent());
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode, XML_MIN_COST_DECREASE_NAME);
		if (elem != null) {
			MIN_COST_DECREASE = Double.parseDouble(elem.getTextContent());
		}

		// compatibility
		elem = XMLUtils.getFirstElementByTagName(xmlnode, HFT.XML_ELEMENT_NAME);
		if (elem != null) {
			Experiment.getInstance().setHFT(new HFT<HFTPayload>(elem));
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode, WordTree.XML_ELEMENT_NAME);
		if (elem != null) {
			Experiment.getInstance().setWordTree(new WordTree<WordTreePayload>(elem));
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode, XML_MAX_CLUSTERS_NAME);
		if (elem != null) {
			maxClusters = Integer.parseInt(elem.getTextContent());
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode, XML_OVERT_QUESTION_PENALTY_NAME);
		if (elem != null) {
			overtQuestionPenalty = Double.parseDouble(elem.getTextContent());
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode,XML_FORCE_HIDDEN_QUESTIONS_NAME);
		if (elem != null) {
			forceHiddenQuestions = Integer.parseInt(elem.getTextContent());
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode,XML_PRUNING_THRESHOLD_NAME);
		if (elem != null) {
			pruningThreshold = Integer.parseInt(elem.getTextContent());
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode, XML_CONTEXT_VAR_METRICS_NAME);
		if (elem != null) {
			String className = elem.getAttribute("class");
			try {
				Class<ContextVariableMetrics> theClass = (Class<ContextVariableMetrics>) Class.forName(className);
				Constructor<ContextVariableMetrics> theConstructor = theClass.getConstructor(Element.class);
				contextVariableMetrics = theConstructor.newInstance(elem);
			} catch (ClassNotFoundException e) {
				e.printStackTrace();
			} catch (SecurityException e) {
				e.printStackTrace();
			} catch (NoSuchMethodException e) {
				e.printStackTrace();
			} catch (IllegalArgumentException e) {
				e.printStackTrace();
			} catch (InstantiationException e) {
				e.printStackTrace();
			} catch (IllegalAccessException e) {
				e.printStackTrace();
			} catch (InvocationTargetException e) {
				e.printStackTrace();
			}
		}
		if (contextVariableMetrics == null) {
			contextVariableMetrics = defaultContextVariableMetrics();
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode, XML_CONTEXT_VAR_STOPPING_RULE_NAME);
		if (elem != null) {
			String className = elem.getAttribute("class");
			try {
				Class<ContextVariableStoppingRule> theClass = (Class<ContextVariableStoppingRule>) Class.forName(className);
				Constructor<ContextVariableStoppingRule> theConstructor = theClass.getConstructor(Element.class);
				contextVariableStoppingRule = theConstructor.newInstance(elem);
			} catch (ClassNotFoundException e) {
				e.printStackTrace();
			} catch (SecurityException e) {
				e.printStackTrace();
			} catch (NoSuchMethodException e) {
				e.printStackTrace();
			} catch (IllegalArgumentException e) {
				e.printStackTrace();
			} catch (InstantiationException e) {
				e.printStackTrace();
			} catch (IllegalAccessException e) {
				e.printStackTrace();
			} catch (InvocationTargetException e) {
				e.printStackTrace();
			}
		}
		if (contextVariableStoppingRule == null) {
			contextVariableStoppingRule = defaultContextVariableStoppingRule();
		}
		
		elem = XMLUtils.getFirstElementByTagName(xmlnode,XML_RANDOMNESS_NAME);
		if (elem != null) {
			randomness = Float.parseFloat(elem.getTextContent());
		}
		

	}
	
	private static ContextVariableMetrics defaultContextVariableMetrics() {
		return  new RelativeReverseConditionalEntropy();
	}
	
	private static ContextVariableStoppingRule defaultContextVariableStoppingRule() {
		return new DummyMetrics();
	}
	
	/*
	public static class DecodingRuntime {
		private final double weights[];
		private final BinaryTree<HistoryTreePayload> nodes[];
		
		private DecodingRuntime(double[] backoff,
				BinaryTree<HistoryTreePayload>[] nodes) {
			this.weights = backoff;
			this.nodes = nodes;
		}

		public final double getWeight(int clusterid) {
			return weights[clusterid];
		}

		public final BinaryTree<HistoryTreePayload> getNode(int clusterid) {
			return nodes[clusterid];
		}

		public final BinaryTree<HistoryTreePayload>[] getNodes() {
			return nodes;
		}
	}
	
	private DecodingRuntime decodingRuntime;
	
	@SuppressWarnings("unchecked")
	private DecodingRuntime buildDecodingRuntime() {
		BinaryTree<HistoryTreePayload> nodes[];
		double backoffWeights[];

		getHistoryTree();
		if (!historyTree.getPayload().top) {
			System.err.printf("The model '%s' lacks top-of-the-tree marker, probably the population step needs to be re-run\n", id);
			System.exit(1);
		}
		int maxClusterId = 0;
		for(BinaryTreeIterator<HistoryTreePayload> i = historyTree.getPreOrderIterator(); i.hasNext(); ) {
			HistoryTreePayload payload = i.next();
			if (payload.clusterid > maxClusterId) {
				maxClusterId = payload.clusterid;
			}
		}

		backoffWeights = new double[maxClusterId + 1];
		nodes = new BinaryTree[maxClusterId + 1];
		for(BinaryTreeIterator<HistoryTreePayload> i = historyTree.getPreOrderIterator(); i.hasNext(); ) {
			BinaryTree<HistoryTreePayload> node = i.nextNode();
			int clusterid = node.getPayload().clusterid;
			nodes[clusterid] = node;
			
			backoffWeights[clusterid] = node.getPayload().backoff;
		}
	
		return new DecodingRuntime(backoffWeights, nodes);
	}
	
	public DecodingRuntime getDecodingRuntime() {
		if (decodingRuntime == null) {
			synchronized(this) {
				if (decodingRuntime == null) {
					decodingRuntime = buildDecodingRuntime();
				}
			}
		}
		return decodingRuntime;
	}
	*/
	
	public String getHistoryTreeBaseFilename() {
		return getId() + "-history-tree";
	}
	public String getHistoryTreeFilename() {
		return getHistoryTreeBaseFilename() + ".ser";
	}
	
	public static BinaryTree<HistoryTreePayload> loadXMLHistoryTree(String filename) {
		File file = new File(filename);
		if (!file.exists() || !file.canRead()) {
			System.err.printf("Can't load history tree from %s\n", filename);
			return null;
		}
		BinaryTree<HistoryTreePayload> tree = null;
		 
		Document doc = null;
		DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
		DocumentBuilder db = null;
		try {
			db = dbf.newDocumentBuilder();
			doc = db.parse(IO.getInputStream(filename));
			
			tree = new BinaryTree<HistoryTreePayload>(doc.getDocumentElement()); 
		}catch(ParserConfigurationException pce) {
			pce.printStackTrace();
			System.exit(1);
		}catch(SAXException se) {
			se.printStackTrace();
		}catch(IOException ioe) {
			ioe.printStackTrace();
		}
		return tree;
	}

	@SuppressWarnings("unchecked")
	public static BinaryTree<HistoryTreePayload> loadSerializedHistoryTree(String filename) {
		BinaryTree<HistoryTreePayload> tree = null;
		try {
	    	ObjectInputStream ois = new ObjectInputStream(IO.getInputStream(filename));
	    	tree = (BinaryTree<HistoryTreePayload>) ois.readObject();
	    	ois.close();
		} catch(IOException e) {
			e.printStackTrace();
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}
    	return tree;
	}
	
	private void loadHistoryTree() {
		String historyTreeBase = getHistoryTreeBaseFilename();
		File f = new File(historyTreeBase + ".ser.gz");
		if (f.isFile()) {
			historyTree = loadSerializedHistoryTree(f.getAbsolutePath());
			return;
		}
		f = new File(historyTreeBase + ".ser");
		if (f.isFile()){
			historyTree = loadSerializedHistoryTree(f.getAbsolutePath());
			return;			
		}
		f = new File(historyTreeBase + ".xml.gz");
		if (f.isFile()){
			historyTree = loadXMLHistoryTree(f.getAbsolutePath());
			return;			
		}
		f = new File(historyTreeBase + ".xml");
		if (f.isFile()){
			historyTree = loadXMLHistoryTree(f.getAbsolutePath());
			return;			
		}
		System.err.printf("Failed to find a history tree %s.*\n", historyTreeBase);
	}
	
	public static void saveXMLHistoryTree(BinaryTree<HistoryTreePayload> tree, String filename) {
		
		Document doc = null;
		DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
		DocumentBuilder db = null;
		try {
			db = dbf.newDocumentBuilder();
			doc = db.newDocument();

			Element elem = tree.createXML(doc);
			doc.appendChild(elem);
			
			try {
				DOMImplementationRegistry registry = DOMImplementationRegistry.newInstance();
				
				DOMImplementationLS impl = 
				    (DOMImplementationLS)registry.getDOMImplementation("LS");
				LSSerializer writer = impl.createLSSerializer();
				writer.getDomConfig().setParameter("format-pretty-print", Boolean.FALSE);
				LSOutput output = impl.createLSOutput();
				output.setEncoding("UTF-8");
				FileWriter fWriter = new FileWriter(filename);
				output.setCharacterStream(fWriter);
				writer.write(doc.getDocumentElement(), output);
				fWriter.close();
			} catch (Exception e) {
				e.printStackTrace();
			}
		}catch(ParserConfigurationException pce) {
			pce.printStackTrace();
			System.exit(1);
		}
		
	}

	public static void saveSerializedHistoryTree(BinaryTree<HistoryTreePayload> tree, String filename) {
    	ObjectOutputStream oos;
		try {
			oos = new ObjectOutputStream(IO.getOutputStream(filename));
	    	oos.writeObject(tree);
	    	oos.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	public void saveHistoryTree() {
		if (historyTree == null) return;
		
		String historyTreeBase = getHistoryTreeBaseFilename();
		saveSerializedHistoryTree(historyTree, historyTreeBase + ".ser.gz");
	}
    /**
	 * @return the hft
	 */
	public HFT<HFTPayload> getHFT() {
		return Experiment.getInstance().getHFT();
	}
	/**
	 * @return the order
	 */
	public byte getOrder() {
		return order;
	}
	
	public byte getOvertOrder() {
		return overtOrder;
	}

	public byte getHiddenOrder() {
		return hiddenOrder;
	}
	
	/* (non-Javadoc)
	 * @see edu.umd.clip.lm.util.XMLizable#createXML(org.w3c.dom.Document)
	 */
	@Override
	public Element createXML(Document doc) {
		Element element = doc.createElement(XML_ELEMENT_NAME);
		//element.setAttribute(XML_ATTR_ORDER_NAME, Byte.toString(order));
		element.setAttribute(XML_ATTR_OVERT_ORDER_NAME, Byte.toString(overtOrder));
		element.setAttribute(XML_ATTR_HIDDEN_ORDER_NAME, Byte.toString(hiddenOrder));
		
		element.setAttribute(XML_ATTR_ID_NAME, id);
		
		/*
		if (decoder == null) {
			//element.appendChild(doc.importNode(getDecoderConfig(), true));
		} else {
			element.appendChild(getDecoder().createXML(doc));
		}
		*/
		
		{
			Element e;
			
			e = doc.createElement(XML_USE_COMPLEX_QUESTIONS_NAME);
			e.setTextContent(Boolean.toString(useComplexQuestions));
			element.appendChild(e);
			
			e = doc.createElement(XML_USE_HFT_QUESTIONS_NAME);
			e.setTextContent(Boolean.toString(useHFTQuestions));
			element.appendChild(e);
			
			e = doc.createElement(XML_MIN_TOTAL_COUNT_NAME);
			e.setTextContent(Long.toString(MIN_TOTAL_COUNT));
			element.appendChild(e);

			e = doc.createElement(XML_MIN_COST_DECREASE_NAME);
			e.setTextContent(Double.toString(MIN_COST_DECREASE));
			element.appendChild(e);

			e = doc.createElement(XML_MAX_CLUSTERS_NAME);
			e.setTextContent(Integer.toString(maxClusters));
			element.appendChild(e);

			e = doc.createElement(XML_OVERT_QUESTION_PENALTY_NAME);
			e.setTextContent(Double.toString(overtQuestionPenalty));
			element.appendChild(e);

			e = doc.createElement(XML_FORCE_HIDDEN_QUESTIONS_NAME);
			e.setTextContent(Integer.toString(forceHiddenQuestions));
			element.appendChild(e);

			e = doc.createElement(XML_PRUNING_THRESHOLD_NAME);
			e.setTextContent(Integer.toString(pruningThreshold));
			element.appendChild(e);
			
			if (contextVariableMetrics == null) {
				contextVariableMetrics = defaultContextVariableMetrics();
			}
			e = doc.createElement(XML_CONTEXT_VAR_METRICS_NAME);
			contextVariableMetrics.saveXML(e);
			element.appendChild(e);
			
			if (contextVariableStoppingRule == null) {
				contextVariableStoppingRule = defaultContextVariableStoppingRule();
			}
			e = doc.createElement(XML_CONTEXT_VAR_STOPPING_RULE_NAME);
			contextVariableStoppingRule.saveXML(e);
			element.appendChild(e);

			e = doc.createElement(XML_RANDOMNESS_NAME);
			e.setTextContent(Float.toString(randomness));
			element.appendChild(e);
		}
		
		if (historyTree != null) {
			saveHistoryTree();
			/*
			Element e = doc.createElement(XML_HISTORY_TREE_NAME);
			e.appendChild(historyTree.createXML(doc));
			element.appendChild(e);
			*/
		}
		
		return element;
	}
	/* (non-Javadoc)
	 * @see edu.umd.clip.lm.util.XMLizable#getElementName()
	 */
	@Override
	public String getElementName() {
		return XML_ELEMENT_NAME;
	}

	/**
	 * @return the wordTree
	 */
	public WordTree<WordTreePayload> getWordTree() {
		return Experiment.getInstance().getWordTree();
	}

	public String getId() {
		return id;
	}

	public BinaryTree<HistoryTreePayload> getHistoryTree() {
		if (historyTree == null) {
			synchronized(this) {
				if (historyTree == null) {
					loadHistoryTree();
				}
			}
		}
		return historyTree;
	}

	public void setHistoryTree(BinaryTree<HistoryTreePayload> historyTree) {
		this.historyTree = historyTree;
	}

	public Element getDecoderConfig() {
		Element lmConfig = Experiment.getInstance().getLMConfig(id);
		Element element = XMLUtils.getFirstElementByTagName(lmConfig, Decoder.XML_ELEMENT_NAME);
		if (element == null) {
			element = lmConfig.getOwnerDocument().createElement(Decoder.XML_ELEMENT_NAME);
			lmConfig.appendChild(element);
		}
		return element;
	}

	/*
	public Decoder getDecoder() {
		if (decoder != null) return decoder;
		synchronized(this) {
			if (decoder == null) {
				decoder = new Decoder(this, getDecoderConfig());
			}
			return decoder;
		}
	}
	*/
	
	public boolean useComplexQuestions() {
		return useComplexQuestions;
	}

	public boolean useHFTQuestions() {
		return useHFTQuestions;
	}

	public long getMIN_TOTAL_COUNT() {
		return MIN_TOTAL_COUNT;
	}

	public double getMIN_COST_DECREASE() {
		return MIN_COST_DECREASE;
	}

	public int getIdNum() {
		return idNum;
	}

	protected void setIdNum(int idNum) {
		this.idNum = idNum;
	}

	public int context2cluster(Context ctx) {
		BinaryTree<HistoryTreePayload> tree = getHistoryTree();
		HistoryTreePayload payload = tree.getPayload();
		int clusterid = payload.clusterid;
		while(true) {
			if (tree == null) break;
			payload = tree.getPayload();
			clusterid = payload.clusterid;
			
			if (payload.question == null) break;
			
			boolean result = payload.question.test(ctx);
			tree = result ? tree.getRight() : tree.getLeft();
		}
		return clusterid;
	}

	public double getOvertQuestionPenalty() {
		return overtQuestionPenalty;
	}

	public void setOvertQuestionPenalty(double overtQuestionPenalty) {
		this.overtQuestionPenalty = overtQuestionPenalty;
	}

	public int getMaxClusters() {
		return maxClusters;
	}

	public void setMaxClusters(int maxClusters) {
		this.maxClusters = maxClusters;
	}

	public int getForceHiddenQuestions() {
		return forceHiddenQuestions;
	}

	public void setForceHiddenQuestions(int forceHiddenQuestions) {
		this.forceHiddenQuestions = forceHiddenQuestions;
	}

	public int getPruningThreshold() {
		return pruningThreshold;
	}

	public void setMIN_TOTAL_COUNT(long min_total_count) {
		MIN_TOTAL_COUNT = min_total_count;
	}

	public void setPruningThreshold(int pruningThreshold) {
		this.pruningThreshold = pruningThreshold;
	}

	public ContextVariableMetrics getContextVariableMetrics() {
		return contextVariableMetrics;
	}

	public void setContextVariableMetrics(
			ContextVariableMetrics contextVariableMetrics) {
		this.contextVariableMetrics = contextVariableMetrics;
	}

	public ContextVariableStoppingRule getContextVariableStoppingRule() {
		return contextVariableStoppingRule;
	}

	public void setContextVariableStoppingRule(
			ContextVariableStoppingRule contextVariableStoppingRule) {
		this.contextVariableStoppingRule = contextVariableStoppingRule;
	}

	public void setMIN_COST_DECREASE(double mINCOSTDECREASE) {
		MIN_COST_DECREASE = mINCOSTDECREASE;
	}

	public static List<LanguageModel> initDecoding(final LMDecodingOptions opts) throws IOException, ClassNotFoundException, DatabaseException {
		Experiment.initialize(opts.config);
		Experiment experiment = Experiment.getInstance();

		experiment.setDebugDecoder(opts.debug);
		
		JobManager.initialize(opts.jobs);
		Thread thread = new Thread(JobManager.getInstance(), "Job Manager");
		thread.setDaemon(true);
		thread.start();
		JobManager manager = JobManager.getInstance();
		
		experiment.buildPrefixes();
		experiment.buildWordPrefixes();

		final ForestModel forest = experiment.getForest(opts.forest);
		
		Experiment.Files files = experiment.getFiles();

		
		//LinkedList<LanguageModel> LMs = new LinkedList<LanguageModel>();
		// load the chain of LMs
		/*
		{
			LanguageModel boLM = forest;
			while(true) {
				// make the list of LMs in reverse order 
				LMs.addFirst(boLM);
				if (boLM.getBackoffLM() == null) break;
				boLM = experiment.getLM(boLM.getBackoffLM());
			}
		}
		*/
		AbstractProbTreeStorage storage = null;

		JobGroup group = manager.createJobGroup("init");
		
		switch(opts.storage) {
		case COMPACT :
		{
			final CompactProbTreeStorage compactStorage = new CompactProbTreeStorage(0);
			final InterpolatingProbTreeStorage interpolatingStorage = new InterpolatingProbTreeStorage(forest, compactStorage);
			interpolatingStorage.setCacheSize(100000);
			storage = interpolatingStorage;
			
			for(final LanguageModel _lm : forest.getModels()) {
				//if (opts.dontOpenCurrentLMStorage && _lm == lm) continue;
				Runnable run2 = new Runnable() {
					@Override
					public void run() {
						compactStorage.openForReading(_lm.getIdNum());
					}
				};
				manager.addJob(group, new Job(run2, _lm.getId()));
			}
		}
		break;
		case REMOTE:
		{
			InetSocketAddress address = new InetSocketAddress(InetAddress.getByName(opts.host), opts.port);
			ClientStorageConnection connection = new ClientStorageConnection(address);
			Timer timer = new Timer(true);
			long interval = ClientStorageConnection.ClientMonitor.INTERVAL;
			timer.scheduleAtFixedRate(new ClientStorageConnection.ClientMonitor(connection), interval, interval);
			
			ClientProbTreeStorage remoteStorage = new ClientProbTreeStorage(connection);
			int hardCacheSize = (int) (3000 * Math.pow(4.0, forest.getOrder()));
			hardCacheSize /= 2;
			
			storage = new CachedProbTreeStorage(remoteStorage, hardCacheSize, hardCacheSize*2);
		}
		break;
		case BDB:
		{
			Environment env = BDBProbTreeStorage.createEnvironment(files.getDb(), false);
			BDBProbTreeStorage dbStorage = new BDBProbTreeStorage(env);
			InterpolatingProbTreeStorage interpolatingStorage = new InterpolatingProbTreeStorage(forest, dbStorage);
			interpolatingStorage.setCacheSize(100000);
			storage = interpolatingStorage;

			for(final LanguageModel _lm : forest.getModels()) {
				//if (opts.dontOpenCurrentLMStorage && _lm == lm) continue;
				dbStorage.open(_lm.getId(), _lm.getIdNum(), false);
			}
		}
		break;
		}
		
		for(final LanguageModel _lm : forest.getModels()) {
			Runnable run1 = new Runnable() {
				@Override
				public void run() {
					_lm.getHistoryTree();
					//if (opts.dontOpenCurrentLMStorage && _lm == lm) return;
					//_lm.getDecoder().setStorage(_storage);
				}
			};
			manager.addJob(group, new Job(run1, _lm.getId()));
			//run1.run();
		}
		group.join();
		
		forest.getDecoder().setStorage(storage);
		
		if (opts.coarseThreshold > 0) {
			Decoder.USE_COARSE_AND_FINE = true;
			Decoder.COARSE_THRESHOLD = opts.coarseThreshold;
		}
		
		DecoderCompactProbTree emptyTree = DecoderCompactProbTree.constructEmptyTree(Experiment.getInstance().getHFT().getTree());
		DecoderCompactProbTree.setEmptyTree(emptyTree);
		
		experiment.closeXML();
		
		return forest.getModels();
	}

	public float getRandomness() {
		return randomness;
	}

	public void setRandomness(float randomness) {
		this.randomness = randomness;
	}
}
